/**
 * 
 */
package test;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.springframework.util.StringUtils;

import com.battcn.platform.service.utils.UserAgentUtil;
import com.battcn.platform.service.webmagic.downloader.CrawlerDownloader;
import com.battcn.platform.service.webmagic.downloader.CrowProxyProvider;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.selector.Html;

/**
 * @author yanglei
 *
 *         2017年12月19日
 */
public class testprocess implements PageProcessor {
	// 导演正则表达式
	public static final String URL_DIRECTOR = "https?://www\\.80s\\.tw/director/\\d+";
	// 演员正则表达式
	public static final String URL_ACTOR = "https?://www\\.80s\\.tw/actor/\\d+";
	private Site site = Site.me()
			// .addHeader("Proxy-Authorization",ProxyGeneratedUtil.authHeader(ORDER_NUM,
			// SECRET, (int) (new Date().getTime()/1000)))//设置代理 相当于账号密码
			.setDisableCookieManagement(true).setCharset("UTF-8").setTimeOut(300000).setRetryTimes(3).setCycleRetryTimes(5)
			.setSleepTime(new Random().nextInt(20) * 100).setUserAgent(UserAgentUtil.getRandomUserAgent());

	/*
	 * (non-Javadoc)
	 * 
	 * @see us.codecraft.webmagic.processor.PageProcessor#process(us.codecraft.
	 * webmagic.Page)
	 */
	@Override
	public void process(Page page) {
		crawlactor(page);
	//	crawldirector(page);
		/*String movieId=getMatcher("\\d+", page.getUrl().toString());
		page.putField("movieId", getMatcher("\\d+", page.getUrl().toString()));
		String type=getMatcher("https?://www.80s.tw/(.+)/\\d+", page.getUrl().toString(), 1);
		String movieDetail = page.getHtml().xpath("//div[@id='body']/div").toString();
		if (!StringUtils.isEmpty(movieDetail)&&new Html(movieDetail).xpath("//div[@class='info']/h1[@class='font14w']/tidyText()") != null) {
			Map<String, String> downLoadMap = new HashMap<>();
			String movieInfo = new Html(movieDetail).xpath("//div[@class='info']").toString();
			page.putField("title", new Html(movieInfo).xpath("//h1[@class='font14w']/tidyText()"));
			page.putField("content", new Html(movieDetail).xpath("//div[@id='movie_content']/tidyText()"));
			//上映日期
			String releaseDate=new Html(movieInfo).xpath("//div[@class='clearfix']/span/tidyText()").regex("上映日期：(.+)",1).toString();
			
			// 解析分集和下载地址
			List<String> downLoads = new Html(movieDetail)
					.xpath("//ul[@class='dllist1']//li[@class='clearfix dlurlelement' OR @class='clearfix dlurlelement backcolor1']//span[@class='dlname nm']//a")
					.all();
			for (String download : downLoads) {
				String title = new Html(download).xpath("//a/text()").toString();
				String downloadurl = new Html(download).xpath("//a/@href").toString();
				downLoadMap.put(title, downloadurl);
			}
			page.putField("downUrls", downLoadMap);
			// 先将span_block元素全部匹配下来 再进行正则匹配 选择正确的一条
			String gradeHtml = new Html(movieInfo).xpath("//div[@class='clearfix']//span[@class='span_block']")
					.regex(".*score sc8.*").toString();
			if(gradeHtml!=null){
				// 这里获取出来的是豆瓣评分：8.3
				String grade = new Html(gradeHtml).xpath("tidyText()").toString();
				// 截取数字
				String regEx = "[^0-9.]";
				Pattern p = Pattern.compile(regEx);
				Matcher m = p.matcher(grade);
				page.putField("grade", m.replaceAll("").trim());
			}
			
			
			// 演员
			List<String> actors = new Html(movieInfo).links().regex("/actor/\\d+").all();
			// 导演
			List<String> directors = new Html(movieInfo).links().regex("/director/\\d+").all();
		}*/

	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see us.codecraft.webmagic.processor.PageProcessor#getSite()
	 */
	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}
	
	private void crawldirector(Page page) {// 抓取导演页
		String director=page.getHtml().xpath("//div[@class='f_block1']/h1/tidyText()").toString().trim();
		String regEx = "\"(.+)\"导演作品专辑";
		page.putField("directorName", getMatcher(regEx, director,1));
		page.putField("directorId",getMatcher("\\d+", page.getUrl().toString()));
		
		List<String> movieList=page.getHtml().xpath("//ul[@class='me1 clearfix']").links().all();
		Map<String, String> movieIdMap = new HashMap<>();
		for(String movie:movieList){
			String movieid=getMatcher("\\d+", movie);
			if(movieIdMap.get(movieid)==null){
				movieIdMap.put(movieid, movieid);
			}
		}
		page.putField("movieIdMap", movieIdMap);
	}
	/**
	 * 演员页信息
	 * 
	 * @param page
	 *            当前页面对象
	 */
	private void crawlactor(Page page) {// 抓取演员页
		String chineseName="中文名：";
		String sex="性别：";
		String homeplace="出生地：";
		// 截取数字
		page.putField("actorId", getMatcher("\\d+", page.getUrl().toString()));
		String actor=page.getHtml().xpath("//div[@class='noborder block1']/div[@class='f_block1']/h1//tidyText()").toString();
		//截取演员姓名
		String regEx = "(.+)电影全集";
		page.putField("actorName", getMatcher(regEx, actor,1));
		
		String actorInfo=page.getHtml().xpath("//div[@id='block3']/div[@class='noborder block1']/div").regex(".*中文名.*").toString();
		
		List<String> movieidList=page.getHtml().xpath("//div[@id='block3']/div[@class='clearfix noborder block1']/ul").links().regex("https?://www.80s.tw/\\w+/(\\d+)", 1).all();
		Map<String, String> movieidMap=new HashMap<>();
		if(movieidList!=null&&movieidList.size()>0){
			for(String movieid:movieidList){
				if(movieidMap.get(movieid)==null){
					movieidMap.put(movieid, movieid);
				}
			}
		}
		page.putField("movieids", movieidMap);
		
		if(actorInfo!=null){
			String[] actorInfos = actorInfo.split("\n"); 
			for(String str:actorInfos){
				if(!org.springframework.util.StringUtils.isEmpty(str)){
					if(str.length()>6&&str.substring(0, 6).trim().equals(chineseName)){  
						str=str.replace(chineseName, "").trim();
						page.putField("chineseName",str);
					}
					if(str.indexOf(sex)!=-1){  
						str=str.replace(sex, "").trim();
						page.putField("sex", str);
					}
					if(str.indexOf(homeplace)!=-1){  
						str=str.replace(homeplace, "").trim();
						page.putField("homeplace", str);
					}
				}
				
			}
		}
		
		
		System.out.println(actorInfo);
	}
	public static void main(String[] args) {
		//49.76.125.71:808 [/49.76.125.71]
		List<Proxy> proxies = new ArrayList<>();
		CrawlerDownloader httpClientDownloader = new CrawlerDownloader();
		// 设置动态转发代理，使用定制的ProxyProvider
		httpClientDownloader.setProxyProvider(
				CrowProxyProvider.from(new Proxy("49.76.125.71", 808), new Proxy("119.115.235.202", 8118),
						new Proxy("112.81.30.60", 8118)/*, new Proxy("27.46.6.171", 9999),
						new Proxy("60.205.125.201", 8118), new Proxy("219.223.251.173", 3128),
						new Proxy("211.159.177.212", 3128), new Proxy("58.220.95.107", 8080)*/));
		  Spider.create(new testprocess()).addUrl("https://www.80s.tw/actor/24524").setDownloader(httpClientDownloader)
		  .addPipeline(new ConsolePipeline()).thread(6).run();
		 
	/*	String url = "<span class='span_block'> <span class='font_888'>豆瓣评分：</span> <span class='score sc8'></span>8.3 </span>";
		String regex = "(?=<span class=\"score sc8\"></span>).*(?=</span>)";
		// String regex = "(?=</span>).*(?=</span>)";
		System.out.println(getMatcher(regex, url));*/

	}

	public static String getMatcher(String regex, String source,Integer goroup) {
		String result = "";
		Pattern pattern = Pattern.compile(regex);
		Matcher matcher = pattern.matcher(source);
		while (matcher.find()) {
			if(goroup!=null){
				result = matcher.group(goroup);
			}else{
				result = matcher.group();
			}
		}
		return result;
	}
	public static String getMatcher(String regex, String source) {
		return getMatcher(regex, source,null);
	}

}
